import java.net.HttpURLConnection;
import java.net.URL;
import java.sql.Date;
import java.util.ArrayList;
import java.util.GregorianCalendar;
import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.beans.StringBean;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.tags.MetaTag;
import org.htmlparser.tags.TitleTag;
import org.htmlparser.util.NodeList;


import entity.WebPage;


public class Downloader {

	
	
	public Downloader ()  {
      	        
	}
	
	public WebPage downloadCrawlerResult(String url) {
		WebPage webPage = null;
		try {
			
			
        	Parser parser = new Parser();
        	parser.setResource(url.toLowerCase());
        	
        	MyFilter filter = new MyFilter();
            NodeList list = parser.parse(filter);
            
            String descriptionOrTitle = "";
            String keywords = "";
            String content = "";
            ArrayList urlList = new ArrayList();
    
            
            
            Node node = null;
            
            for(int i = 0; i < list.size(); i++) {
            	node = list.elementAt(i);
            	//check if note is a metatag
	            if (node instanceof MetaTag) {
	                MetaTag meta = (MetaTag) node;
	                //if it is not null
	                if(meta.getMetaTagName() != null) {
	                	//we want to capture the meta data of description
		                if(meta.getMetaTagName().equalsIgnoreCase("description"))
		                	descriptionOrTitle = meta.getAttribute("content");
		                //else if the meta data is a keyword attribute, then we capture the keywords content
		                else if (meta.getMetaTagName().equalsIgnoreCase("keywords"))
		                	keywords = meta.getAttribute("content");
	                }
	            }
	            
	            //if node is a title tag
	            else if (node instanceof TitleTag)  {
	            	//check if meta data description of the website is found
	            	if(descriptionOrTitle.length() == 0) {
	            	//if description is not there, then we use title of the webpage to replace it
	            		TitleTag title = (TitleTag) node;
	            		descriptionOrTitle = (title.getTitle()).replace("\n", "");
	            	}
	            }
	            
	            //if node is a link tag
	            else if (node instanceof LinkTag) {
	            	LinkTag link = (LinkTag) node;
	            	String address = link.extractLink();	            	
	            	if(address.contains("http") && !address.contains("#"))
	            		urlList.add(address);
	            }
            }//end for
            
            
            StringBean sb = new StringBean();
            sb.setURL(url);
            parser.visitAllNodesWith(sb);
            content = sb.getStrings();
            
            webPage = new WebPage(url.toLowerCase(), keywords, descriptionOrTitle, content, urlList);
            
       
        } catch (Exception e) {
            System.out.println("\nERROR parsing website : "+url);
        }
        
        return webPage;
	}
	
	

	
}
